package cn.edu.buaa.cnsatm.traffic_big_data_server.util;
import com.sun.rowset.internal.Row;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.Connection;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.xml.ws.http.HTTPException;
import java.io.IOException;


import java.io.*;
import java.net.*;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
//import org.apache.poi.hssf.usermodel.HSSFWorkbook;
//import org.apache.poi.ss.usermodel.Cell;
//import org.apache.poi.ss.usermodel.Sheet;
//import org.apache.poi.ss.usermodel.Workbook;
//import org.apache.poi.xssf.usermodel.XSSFWorkbook;

/**
 * @name = LX
 * @classname = WebSpiderUtil
 * @description = 存放各种爬虫通用工具
 * @date = 2019/10/18
 */
public class WebSpiderUtil {
    public static void main(String args[]) throws IOException {
        getExcelFromUrl("portTemp","http://www.shipxy.com/Ship/ExportAtPortShips?points=117723139.38875288,39047250,117936426.61124712,39047250,117936426.61124712,38881250,117723139.38875288,38881250", "_elane_maptype=MT_SEA; Hm_lvt_adc1d4b64be85a31d37dd5e88526cc47=1572776664,1572776970; tc_TC=; _elane_shipfilter_type=%u8D27%u8239%2C%u96C6%u88C5%u7BB1%u8239%2C%u6CB9%u8F6E%2C%u5F15%u822A%u8239%2C%u62D6%u8F6E%2C%u62D6%u5F15%2C%u6E14%u8239%2C%u6355%u635E%2C%u5BA2%u8239%2C%u641C%u6551%u8239%2C%u6E2F%u53E3%u4F9B%u5E94%u8239%2C%u88C5%u6709%u9632%u6C61%u88C5%u7F6E%u548C%u8BBE%u5907%u7684%u8239%u8236%2C%u6267%u6CD5%u8247%2C%u5907%u7528-%u7528%u4E8E%u5F53%u5730%u8239%u8236%u7684%u4EFB%u52A1%u5206%u914D%2C%u5907%u7528-%u7528%u4E8E%u5F53%u5730%u8239%u8236%u7684%u4EFB%u52A1%u5206%u914D%2C%u533B%u7597%u8239%2C%u7B26%u540818%u53F7%u51B3%u8BAE%28Mob-83%29%u7684%u8239%u8236%2C%u62D6%u5F15%u5E76%u4E14%u8239%u957F%3E200m%u6216%u8239%u5BBD%3E25m%2C%u758F%u6D5A%u6216%u6C34%u4E0B%u4F5C%u4E1A%2C%u6F5C%u6C34%u4F5C%u4E1A%2C%u53C2%u4E0E%u519B%u4E8B%u884C%u52A8%2C%u5E06%u8239%u822A%u884C%2C%u5A31%u4E50%u8239%2C%u5730%u6548%u5E94%u8239%2C%u9AD8%u901F%u8239%2C%u5176%u4ED6%u7C7B%u578B%u7684%u8239%u8236%2C%u5176%u4ED6; _elane_shipfilter_length=0%2C40%2C41%2C80%2C81%2C120%2C121%2C160%2C161%2C240%2C241%2C320%2C321%2C9999; _elane_shipfilter_sog=0%2C1; _elane_shipfilter_country=0%2C1; _filter_flag=-1; _elane_shipfilter_olength=0; tc_QX=; leadPageCookie=1; bd0426=%128%18%19%14%276%18; FD857C2AF68165D4=Tr9r7iAhRNkpP9X6aYiU1ne7Ud5CQfjCflX9XC0LACRDkqdvG4Aj5RABvtu5KtVo6NJsFlCVYxo=; .UserAuth2=F953329294D9BAD98C74C28C11D8AAEBD9CE8D64959B6DE385C8D136FDB158FBEB21A2216DB171C636BFD7140CA3CDC24695FB324AA249308D4281137FFE0C738D5FF1AFBB1677A16C9FD43320965A21D2E9205640022B4BE7030759BBB155EA06FAFA762224C9113F6E6D8F234A586192156BC3; SERVERID=6462d953552c3e70030b707fb585b257|1573716557|1573716549");
        //TODO:将GEOM转成多边形经纬度点
        //System.out.println(readLocalXlsx("portTemp"));
    }
    public static void getExcelFromUrl(String fileName, String urlString, String cookie) throws IOException {
        /**
         * @name = LX
         * @classname = getExcelFromUrl
         * @description = 从URL保存XLSX
         * @date = 2019/11/14
         */
        URL url = new URL(urlString);
        HttpURLConnection conn = (HttpURLConnection ) url.openConnection();
        conn.addRequestProperty("Cookie", cookie);
        conn.connect();
        // 打印HTTP header
        Map headers = conn.getHeaderFields();
        Set<String> keys = headers.keySet();
        // 转换编码
        String contentDisposition = URLDecoder.decode(
                conn.getHeaderField("content-Disposition"), "UTF-8");
        //System.out.println(contentDisposition);
        // 匹配文件名
        Pattern pattern = Pattern.compile(".*fileName=(.*)");
        Matcher matcher = pattern.matcher(contentDisposition);
        //System.out.println(matcher.groupCount());
        //System.out.println(matcher.matches());
        String filename = contentDisposition.split("=")[1];
        RandomAccessFile file = new RandomAccessFile(fileName + ".xlsx", "rw");
        InputStream stream = conn.getInputStream();
        byte buffer[] = new byte[1024];
        while (true) {
            int len = stream.read(buffer);
            if (len == -1) {
                break;
            }
            file.write(buffer, 0, len);
        }
        if (file != null) {
            file.close();
        }
        if (stream != null) {
            stream.close();
        }
        conn.disconnect();
    }

    public static String getJsonFromURL(String url){
        /**
         * @name = LX
         * @classname = getHTMLDocument
         * @description = 从URL中读取json文件
         * @date = 2019/10/18
         */
        StringBuilder json = new StringBuilder();
        try {
            URL urlObject = new URL(url);
            URLConnection uc = urlObject.openConnection();
            uc.connect();
            BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream(),"UTF-8"));
            String inputLine = null;
            while ( (inputLine = in.readLine()) != null) {
                json.append(inputLine);
            }
            in.close();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            return "";
            //e.printStackTrace();
        }
        return json.toString();
    }
    public static String getJsonFromURL(String url, String cookie){
        /**
         * @name = LX
         * @classname = getHTMLDocument
         * @description = 从URL中读取json文件
         * @date = 2019/10/18
         */
        StringBuilder json = new StringBuilder();
        try {
            URL urlObject = new URL(url);
            URLConnection uc = urlObject.openConnection();
            uc.addRequestProperty("Cookie", cookie);
            uc.addRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0");
            BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream(),"UTF-8"));
            String inputLine = null;
            while ( (inputLine = in.readLine()) != null) {
                json.append(inputLine);
            }
            in.close();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            return "";
            //e.printStackTrace();
        }
        return json.toString();
    }
    public static Document getHTMLDocument(String url) throws InterruptedException {
        /**
         * @name = LX
         * @classname = getHTMLDocument
         * @description = 将url里的html文档爬到本地
         * @date = 2019/10/18
         */
        Document document = null;
        try {
            Connection connect = Jsoup.connect(url).ignoreContentType(true);
            document = connect.get();
        }
        catch (HTTPException | IOException e){
            //e.printStackTrace();
        }
        Thread.sleep(500);
        return document;
    }
    public static void saveImageFromWebToDir(String imageURL, String imageName) throws IOException {
        /**
         * @name = LX
         * @classname = saveImageFromWebToDir
         * @description = 从URL中下载图片到指定位置
         * @date = 2019/10/29
         */
        URL url = new URL(imageURL);
        URLConnection con = url.openConnection();// 打开连接
        InputStream is = con.getInputStream();// 输入流
        byte[] bs = new byte[1024];// 1K的数据缓冲
        int len;// 读取到的数据长度
        File file = new File(imageName);
        if(file.exists() && file.isFile()){file.delete();}//防止文件重复写
        FileOutputStream os = new FileOutputStream(file, true);// 输出的文件流
        while ((len = is.read(bs)) != -1) { os.write(bs, 0, len); }
        os.close();// 完毕，关闭所有链接
        is.close();
    }
    //    protected static int readLocalXlsx(String fileName) throws IOException {
//        /**
//         * @name = LX
//         * @classname = readLocalXlsx
//         * @description = 读取本地XLSX文件
//         * @date = 2019/11/14
//         */
//        try {
//            //String encoding = "GBK";
//            File excel = new File(fileName + ".xlsx");
//            if (excel.isFile() && excel.exists()) {
//                Workbook wb = new XSSFWorkbook(excel);
//                Sheet sheet = wb.getSheetAt(0);
//                int shipNum =sheet.getLastRowNum() - 1;
//                //excel.delete();
//                return shipNum;
//            } else {
//                System.out.println("找不到指定的文件");
//            }
//        } catch (Exception e) {
//            e.printStackTrace();
//        }
//        return -1;
//    }
    public static boolean delFile(File file) {
        /**
         * @name = LX
         * @classname = delFile
         * @description = 删除文件，若指定的是目录则删除目录下文件
         * @date = 2019/11/14
         */
        if (!file.exists()) {
            return false;
        }

        if (file.isDirectory()) {
            File[] files = file.listFiles();
            for (File f : files) {
                delFile(f);
            }
        }
        return file.delete();
    }
    public static void saveJsonToLocalDir(String dir, String fileName, String jsonString){
        BufferedWriter writer = null;
        PropertiesUtil.setProperties();
        File myPath = new File(dir);
        if (!myPath.exists()){//只能建立一级目录
            myPath.mkdir();
            System.out.println("创建文件夹路径为："+ dir);
        }

        File file = new File(dir + fileName + ".json");
        //如果文件不存在，则新建一个
        if(!file.exists()){
            try {
                file.createNewFile();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        else{
            return ;
        }
        //写入
        try {
            writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file,false), "UTF-8"));
            writer.write(jsonString);
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                if(writer != null){
                    writer.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        System.out.println("json创建成功！");
    }
    public static String readJsonFile(String fileName) {
        String jsonStr = "";
        try {
            File jsonFile = new File(fileName);
            FileReader fileReader = new FileReader(jsonFile);

            Reader reader = new InputStreamReader(new FileInputStream(jsonFile),"utf-8");
            int ch = 0;
            StringBuffer sb = new StringBuffer();
            while ((ch = reader.read()) != -1) {
                sb.append((char) ch);
            }
            fileReader.close();
            reader.close();
            jsonStr = sb.toString();
            return jsonStr;
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
}
